from collections import defaultdict, Counter
import gzip, json, os, math

print "loading ratings"
PROD_count = defaultdict(Counter)
orfile = gzip.open("Ratings.json.gz",'r')
buf = orfile.readline()
PROD = 0
while buf != "":
	PROD += 1
	temp = json.loads(buf)
	PROD_count[temp[1]][temp[0]] = int(float(temp[2]))
	buf = orfile.readline()
orfile.close()

def filename_of(doc_id):
	return "TFMATRIX/" + str(doc_id) + ".json.gz"

print "loading IDF matrix"
IDF = Counter(json.loads(gzip.open("IDFmatrix.json.gz",'r').readline()))
#VOCABULARY = sorted(IDF.keys())

outfile = gzip.open("Weights.json.gz",'w')
scorefile = gzip.open("Scores.json.gz",'w')

CORPUS = sorted(set([w.split(".")[0] for w in os.listdir("TFMATRIX")]) & set(PROD_count.keys()))

print "creating the two matrixes needed for rocchio"
outfile.write(json.dumps(CORPUS)+'\n')
scorefile.write(json.dumps(CORPUS)+'\n')
iterator = 0
for (filename,product) in [(filename_of(document),document) for document in CORPUS]:
	iterator += 1
	print "working on product: ", iterator ," out of ", PROD
	FREQ = Counter(json.loads(gzip.open(filename,'r').readline()))
	#print FREQ
	TF = Counter()
	RF = [math.pow(TF[w],2)*math.log10(math.pow(math.pow(10,IDF[w]),2)) for w in FREQ]
	print RF
	max_freq = float(max(FREQ.values()))
	c = math.sqrt(sum(RF))
	print "document weight: ", c
	#for word in VOCABULARY: # this will create vectors!
	for word in FREQ: # only words in document
		TF[word] = FREQ[word] / max_freq
		WEIGHT[word] = ((TF[word]*IDF[word])/ c )
	outfile.write(json.dumps(WEIGHT)+'\n')
	scorefile.write(json.dumps(PROD_count[product])+'\n')
outfile.close()
scorefile.close()

# 210219  scorings from  61  users about  130895  books









